{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from selenium import webdriver \n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-15-89ffedf7a14e>:7: DeprecationWarning: use options instead of chrome_options\n",
      "  driver = webdriver.Chrome( chrome_options = opts)\n"
     ]
    }
   ],
   "source": [
    "opts = webdriver.ChromeOptions()\n",
    "opts.add_argument('--no-sandbox')#解决DevToolsActivePort文件不存在的报错\n",
    "opts.add_argument('window-size=1920x3000') #指定浏览器分辨率\n",
    "opts.add_argument('--disable-gpu') #谷歌文档提到需要加上一这个属性来规避bug\n",
    "opts.add_argument('--hide-scrollbars') #隐藏滚动条, 应对些特殊页面\n",
    "#opts.add_argument('blink-settings=imagesEnabled=false') #不加载图片, 提升速度\n",
    "driver = webdriver.Chrome( chrome_options = opts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from urllib.parse import urljoin\n",
    "from uuid import uuid4\n",
    "import requests\n",
    "import base64\n",
    "from PIL  import Image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 调用api"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def base64_api(uname, pwd, img, typeid):\n",
    "    with open(img, 'rb') as f:\n",
    "        base64_data = base64.b64encode(f.read())\n",
    "        b64 = base64_data.decode()\n",
    "    data = {\"username\": uname, \"password\": pwd, \"typeid\": typeid, \"image\": b64}\n",
    "    result = json.loads(requests.post(\"http://api.ttshitu.com/predict\", json=data).text)\n",
    "    if result['success']:\n",
    "        return result[\"data\"][\"result\"]\n",
    "    else:\n",
    "        return result[\"message\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cut_picture(picture_path):\n",
    "    srcPath = picture_path+\".jpg\"\n",
    "    dstPath = picture_path+\".png\"\n",
    "\n",
    "    # 读取图片\n",
    "    img_1 = Image.open(srcPath)\n",
    "    # 设置裁剪的位置\n",
    "    crop_box = (486,354,750,415)\n",
    "    # 裁剪图片\n",
    "    img_2 = img_1.crop(crop_box)\n",
    "    img_2.save(dstPath)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 判断文章是否下载成功"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "419\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "PATH = r\"D:\\大二下\\数据挖掘\\cnki_mining\\pdf\"\n",
    "FILE_COUNT = len(os.listdir(PATH)) # 全局变量\n",
    "print(FILE_COUNT)\n",
    "\n",
    "def get_file_count():\n",
    "    files = os.listdir(PATH)\n",
    "    file_count = len(files)\n",
    "    return file_count\n",
    "\n",
    "def is_success(title,count):\n",
    "    global FILE_COUNT\n",
    "    print(f\"{title}正在进行第{count+1}次尝试\")\n",
    "    time.sleep(4) # 睡眠时间用来下载文件\n",
    "    file_count = get_file_count()\n",
    "    if file_count - 1 == FILE_COUNT: # 下载成功\n",
    "        print(\"文件下载成功\")\n",
    "        FILE_COUNT = file_count\n",
    "        return True\n",
    "    elif count == 1: # 经过第二轮，还没有下载完，那就不下载了\n",
    "        print(\"文件下载失败\")\n",
    "        return False\n",
    "    else: # 文件下载失败或者文件还没有下载完,再等待4秒进行下载\n",
    "        count += 1\n",
    "        is_success(title,count) # 递归"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 验证码填入过程\n",
    "![code](./pic/ver.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "数据智能:趋势与挑战 需要验证码\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\86188\\anaconda3\\lib\\site-packages\\selenium\\webdriver\\remote\\webdriver.py:1030: UserWarning: name used for saved screenshot does not match file type. It should end with a `.png` extension\n",
      "  warnings.warn(\"name used for saved screenshot does not match file \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "验证码为： e6w2\n",
      "数据智能:趋势与挑战正在进行第1次尝试\n",
      "文件下载成功\n",
      "数据智能:趋势与挑战>>>>>>>>>已下载\n",
      "计算教育学:研究动态与应用场景正在进行第1次尝试\n",
      "文件下载成功\n",
      "计算教育学:研究动态与应用场景>>>>>>>>>已下载\n",
      "Alert Text: 没有订购该产品，请您与贵单位管理员联系订购。\n",
      "Message: unexpected alert open: {Alert text : 没有订购该产品，请您与贵单位管理员联系订购。}\n",
      "  (Session info: chrome=91.0.4472.101)\n",
      "\n",
      "基于关联规则的综合医院评审评价数据挖掘及管理启示----没有下载权限\n",
      "基于频繁时序挖掘的移动政务场景化服务策略研究正在进行第1次尝试\n",
      "文件下载成功\n",
      "基于频繁时序挖掘的移动政务场景化服务策略研究>>>>>>>>>已下载\n",
      "国内基于大数据的信息推荐研究进展:核心内容正在进行第1次尝试\n",
      "文件下载成功\n",
      "国内基于大数据的信息推荐研究进展:核心内容>>>>>>>>>已下载\n",
      "罗颂平治疗肾虚痰湿型多囊卵巢综合征中药周期序贯疗法规律探讨正在进行第1次尝试\n",
      "文件下载成功\n",
      "罗颂平治疗肾虚痰湿型多囊卵巢综合征中药周期序贯疗法规律探讨>>>>>>>>>已下载\n",
      "华东理工大学开发高性能聚合物设计新法正在进行第1次尝试\n",
      "文件下载成功\n",
      "华东理工大学开发高性能聚合物设计新法>>>>>>>>>已下载\n",
      "辅助改善记忆中药类保健食品和中成药组方规律及特点分析正在进行第1次尝试\n",
      "文件下载成功\n",
      "辅助改善记忆中药类保健食品和中成药组方规律及特点分析>>>>>>>>>已下载\n",
      "基于WSR-RDJF国防科技战略情报研究的方法框架正在进行第1次尝试\n",
      "文件下载成功\n",
      "基于WSR-RDJF国防科技战略情报研究的方法框架>>>>>>>>>已下载\n",
      "基于隐结构模型的古代痹证辨证分型及用药规律研究正在进行第1次尝试\n",
      "文件下载成功\n",
      "基于隐结构模型的古代痹证辨证分型及用药规律研究>>>>>>>>>已下载\n"
     ]
    }
   ],
   "source": [
    "data = pd.read_excel(\"CNKI_数据挖掘.xlsx\")\n",
    "base = \"https://kns.cnki.net/\"\n",
    "count,end_count = 550,560\n",
    "title_list = data[\"标题\"].to_list()[count:end_count]\n",
    "download_url_list = data[\"链接\"].to_list()[count:end_count]\n",
    "for title,download_url in zip(title_list,download_url_list):\n",
    "    try:\n",
    "        driver.get(download_url)\n",
    "        time.sleep(0.5)\n",
    "        js = 'document.getElementById(\"pdfDown\").setAttribute(\"target\",\"\");'\n",
    "        driver.execute_script(js)\n",
    "        time.sleep(0.5)\n",
    "        driver.find_element_by_xpath('//li[@class=\"btn-dlpdf\"]').click()\n",
    "        time.sleep(0.5)\n",
    "        source = driver.page_source\n",
    "        if \"安全验证\" in source: # 如果安全验证在html内，那么就需要调用api\n",
    "            print(title,\"需要验证码\")\n",
    "            time.sleep(2)\n",
    "            picture_name = f\"./code/{uuid4()}\"\n",
    "            driver.get_screenshot_as_file(picture_name+\".jpg\")\n",
    "            cut_picture(picture_name)\n",
    "            result = base64_api(uname='', pwd='', img=picture_name+\".png\", typeid=3)\n",
    "            print(\"验证码为：\",result) # 识别出的图片结果\n",
    "            driver.find_element_by_id('vcode').send_keys(result) # 将图片结果填入\n",
    "            time.sleep(5)\n",
    "            driver.find_element_by_xpath('/html/body/div/form/dl/dd/button').click() # 点击提交进行下载\n",
    "            if is_success(title,0): # 如果下载成功\n",
    "                print(f'{title}>>>>>>>>>已下载')\n",
    "            else: # 没有下载成功\n",
    "                print(\"下载失败，换下一个论文进行下载\")\n",
    "        else:\n",
    "            if is_success(title,0): # 如果下载成功\n",
    "                print(f'{title}>>>>>>>>>已下载')\n",
    "            else: # 没有下载成功\n",
    "                print(\"下载失败，换下一个论文进行下载\")\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        print(f'{title}----没有下载权限')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 结果\n",
    "![result](./pic/result.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 可视化"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![](./pic/view.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
